package learning.data;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import learning.data.document.SequenceDocument;
import learning.util.SparseVector;

public class SequenceDataReader {

	static String file = "O:/unix/projects/pardosa/data10/raphaelh/learning/birth_date";
	
	public static boolean readOnlySequencesWithExtractions = false;
	public static boolean learnTransitionFeatures = true; //false;
	
	public static Dataset<SequenceDocument> read(String file) throws IOException {
		return read(file, "utf-8");
	}
	
	public static Dataset<SequenceDocument> read(String file, String charset) throws IOException {
		return read(file, charset, -1);
	}	

	public static Dataset<SequenceDocument> read(String file, String charset, int maxSize) throws IOException {
		return read(new InputStreamReader(new FileInputStream(file + ".data"), charset), 
				    new InputStreamReader(new FileInputStream(file + ".labels"), charset), 
				    new InputStreamReader(new FileInputStream(file + ".features"), charset), maxSize);
	}
	
	@SuppressWarnings("unchecked")
	public static Dataset<SequenceDocument> read(Reader isData, Reader isLabels, Reader isFeatures, int maxSize) throws IOException {
	
		Dataset<SequenceDocument> dataset;
		{
			HashMap<String, Integer> featureIds;
			String[] featureNames;			
			HashMap<String, Integer> labelIds;
			String[] labelNames;
			List<SequenceDocument> docs;
			int[][] transitionFeatures = null;
			
			{
				// read labels
				labelIds = new HashMap<String, Integer>();				
				BufferedReader br = new BufferedReader(isLabels);
				String line = null;
				while ((line = br.readLine()) != null) {
					labelIds.put(line.substring(line.indexOf("\t") + 1), 
							Integer.parseInt(line.substring(0, line.indexOf("\t"))));				
				}
				labelNames = new String[labelIds.size()];
				for (Map.Entry<String, Integer> e : labelIds.entrySet())
					labelNames[e.getValue()] = e.getKey();
				br.close();
			}
			{
				// read features
				featureIds = new HashMap<String, Integer>();
				BufferedReader br = new BufferedReader(isFeatures);
				String line = null;
				while ((line = br.readLine()) != null) {
					// ignore error lines
					if (line.indexOf("\t") < 0) continue;
					
					String val = line.substring(line.indexOf("\t") + 1);
					
					// workaround: there should not be duplicates
					while (featureIds.containsKey(val)) {
						val = Math.random() + "";
					}
					//	System.out.println("contains " + line.substring(line.indexOf("\t") + 1) + " " + line);
					
					featureIds.put(val, Integer.parseInt(line.substring(0, line.indexOf("\t"))));
				}
				
				if (learnTransitionFeatures) {
					transitionFeatures = new int[labelNames.length+1][labelNames.length+1];
					for (int i=0; i < labelNames.length+1; i++)
						for (int j=0; j < labelNames.length+1; j++) {
							int id = featureIds.size();
							transitionFeatures[i][j] = id;
							//System.out.println("adding transition: " + "_transition_" + i + "_" + j);
							featureIds.put("_transition_" + i + "_" + j, id);
						}
				}
					
				// TEMPORARY :
				featureNames = null;
				//featureNames = new String[featureIds.size()];
				//for (Map.Entry<String, Integer> e : featureIds.entrySet())
				//	featureNames[e.getValue()] = e.getKey();
				br.close();
			}
			{
				// read dataset
				docs = new ArrayList<SequenceDocument>();
				BufferedReader br = new BufferedReader(isData);
				List<SparseVector> features = new ArrayList<SparseVector>();
				List<String> tokens = new ArrayList<String>();
				List<Integer> labels = new ArrayList<Integer>();
				HashMap<String,String> meta = new HashMap<String,String>();
				
				String line = null;
				while ((line = br.readLine()) != null) {
					if (maxSize >= 0 && docs.size() >= maxSize) break;
					
					//System.out.println("reading: " + line);
					if (line.length() == 0) {
						
						if (!readOnlySequencesWithExtractions || labels.contains(1))
						{
							if (tokens.size() > 0) {
								
								SequenceDocument doc = new SequenceDocument(tokens.toArray(new String[0]),
										features.toArray(new SparseVector[0]),
										toIntArray(labels));
								if (meta.size() > 0)
									doc.meta = (HashMap<String,String>)meta.clone();
								docs.add(doc);
							}
						}
						features.clear();
						tokens.clear();
						labels.clear();
						meta.clear();
						continue;
					}
					if (line.startsWith("[ ") && line.endsWith(" ]")) {
						String nam = line.substring(2, line.indexOf(":"));
						String val = line.substring(line.indexOf(":") + 1, line.length() - 2);
						meta.put(nam, val);
						continue;
					}
					String[] cols = line.split(" ");
					// workaround for bug with charsets
					if (cols.length==1) {
						line = line += br.readLine();
						cols = line.split(" ");
					}
					
					try {
						tokens.add(cols[0]);
						
						labels.add(Integer.parseInt(cols[1]));
					
						int[] fids = new int[cols.length-2 + (learnTransitionFeatures? 1 : 0)];
						float[] fvals = new float[cols.length-2 + (learnTransitionFeatures? 1 : 0)];
						for (int i=0; i < cols.length-2; i++) {
							String[] pair = cols[i+2].split(":");
							
							fids[i] = Integer.parseInt(pair[0]);
							fvals[i] = Float.parseFloat(pair[1]);
						}
						
						if (learnTransitionFeatures) {
							int startLabel = labels.size() > 1? labels.get(labels.size()-2) : labelNames.length;
							int endLabel = labels.get(labels.size()-1);
							//System.out.println("transition " + startLabel + " - " + endLabel + " [" + transitionFeatures[startLabel][endLabel]);
							fids[fids.length-1] = transitionFeatures[startLabel][endLabel];
							fvals[fids.length-1] = 1.0f;
						}
						
						features.add(new SparseVector(fids, fvals, fids.length));
					} catch (Exception e) { System.out.println("error reading line: " + line); 
						while (line != null && line.length() > 0) line = br.readLine();
						features.clear();
						tokens.clear();
						labels.clear();
						//meta.clear();
						continue;
					}

				}
				br.close();
			}
			dataset = new Dataset<SequenceDocument>(docs.toArray(new SequenceDocument[0]),
					featureNames, featureIds, labelNames, labelIds);
		}
		return dataset;
	}
		
	private static int[] toIntArray(List<Integer> list) {
		int[] ia = new int[list.size()];
		for (int i = 0; i < list.size(); i++) ia[i] = list.get(i);
		return ia;
	}
}
